# -*- coding: utf-8 -*-

import glob
import codecs
import string
import re
import d15n_routines as r

# binary end of line to be used
n = '\x0D\x0A'

d = open('#cleanup_birthnumber_dropped_lines.dat', 'wb')
counters = {'cnt_all_lines':0, 'cnt_ND':0, 'cnt_EMPTY':0}

for f in glob.glob('POC_ID_birthnumber_v3.csv'):
    print f
    i = open(f, 'rb')
    o = open('_cleanup_' + f[:-4] + '.csv', 'wb')

    counters['cnt_all_lines'] = 0
    for l in i.readlines():
        counters['cnt_all_lines'] += 1
        l = r.remove_eol(l)

        if len(l) == 0:
            # skip empty rows
            d.write(f + ' -- ' + l + n)
            pass
        elif counters['cnt_all_lines'] == 1: # skip headers
            pass
            d.write(f + ' -- ' + l + n)
        else:
            # remove multiple spaces, comas
            l = re.sub('[ ,]', '', l)
            l = r.ascii_upper(l)
            cols = string.split(l, '|')

            #print l
            id = cols[0]; bn = cols[1]
            id = id.strip(); bn = bn.strip()

            if bn == '#ND':
                counters['cnt_ND'] += 1
            elif bn in (''):
                counters['cnt_EMPTY'] += 1

            res = "%s\t%s" % (id, bn)

            o.write(res + n)
    i.close(); o.close()
d.close()

print "counters['cnt_all_lines']", str(counters['cnt_all_lines'])
print "counters['cnt_ND']", str(counters['cnt_ND'])
print "counters['cnt_EMPTY']", str(counters['cnt_EMPTY'])
